import requests
from lxml import etree
import DBUtil
import json
from concurrent.futures import ThreadPoolExecutor
import Sha1Util
import time
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
}
def biaoti():
    '''
    爬取二手房的标题和链接，总计，平方价
    :return:
    '''
    sql = "select url,name,chengshi_id from `quyu` "
    result = DBUtil.selectTable(sql)
    for z in result:
        url = z[0]
        wangzhan = requests.get(url=url, headers=headers)
        cun = wangzhan.text
        tree = etree.HTML(cun)
        yema_url = tree.xpath('//*[@id="content"]/div[1]/div[7]/div[2]/div/@page-data')
        if not yema_url:
            continue
        yj = json.loads(yema_url[0])
        for i in range(1, yj["totalPage"]):
            href = url + f'pg{i}'
            print(href)# 每一页的链接
            name = z[1]
            wangzhan = requests.get(url=href, headers=headers)
            cun = wangzhan.text
            tree = etree.HTML(cun)
            biaoti=tree.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[1]/a/text()')
            zongjia=tree.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[6]/div[1]/span/text()')
            pingfang=tree.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[6]/div[2]/span/text()')
            jieshao=tree.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[3]/div/text()')
            biaoti_url=tree.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[1]/a/@href')
            l=len(biaoti)
            for i in range(l):
                a=biaoti[i]
                b=zongjia[i]
                c=biaoti_url[i]
                d=jieshao[i]
                e=pingfang[i]
                b=b+'万'
                id = Sha1Util.jia_mi(a + c)
                sql = '''INSERT into `biaoti` (id,quyu,name,url,jieshao,zongjia,pingfang) VALUES ('%s','%s','%s','%s','%s','%s','%s')
                                                    ''' % (id,name, a, c,d,b,e)
                DBUtil.updateTable(sql)
            time.sleep(0.5)


def duoxiancheng():
    try:
        with ThreadPoolExecutor(20) as th:
            th.submit(biaoti())
            time.sleep(0.5)
    except Exception as e:
        print(e)


duoxiancheng()